#!/usr/bin/env python
# coding=utf-8

# 上海航空国际旅游
import os, sys, re, traceback, logging
import urllib,httplib
import time
import datetime
import getopt
import urllib2
from urllib2 import URLError
from Route  import *


mainBlock_re = re.compile(r'''
        精确查询,请输入关键词查询.*?(<table\b.*?</table>)
        ''', re.X|re.S)

vacationBlock_re = re.compile(r'''
       ((?:<tr>.*?</tr>\s*){3})
        ''', re.X|re.S)

def do_crawl():
    try:
        server_urls = [('跟团游', 'http://www.dreams-travel.com/index/search_find.asp')]
        
        url_pre = 'http://www.dreams-travel.com/index/search_find.asp?typenm=%CF%DF%C2%B7&findnr=&id=&cfid=&classid_about=&classid=&page='
        
        for vacation_type, server_url in server_urls:
            
            html = urllib2.urlopen(server_url).read()
            html = html.decode("gb18030",'ignore').encode('utf-8')
            
            #获取总页数
            pageNums = int(re.compile(r'''1页/(\d{1,3})页''', re.X | re.S).search(html).group(1))
            #根据总页数，遍历所有线路列表页
            for pageNo in range(0, pageNums):
                url = url_pre + str(pageNo+1)
                print url
                html = urllib2.urlopen(url).read()
                html = html.decode("gb18030",'ignore').encode('utf-8')
                
                #获取有用的块
                mainblock = mainBlock_re.findall(html)[0]
        
                #获取所有描述线路的HTML片段 
                vacationBlockList = vacationBlock_re.findall(mainblock)
            
                for div in vacationBlockList:
                    route = Route()
                    try:
                        #站点ID
                        route.site = 27
                        #提供商名称
                        route.supplier = '成都神州国际旅行社 '
                        #供应商url
                        route.supplier_url = 'http://www.dreams-travel.com/'
                    
                        #线路标题
                        route.title = re.compile(r'''title="([^"]*?)"''', re.X|re.S).search(div).group(1).strip()
                        #线路url
                        route.rout_url = 'http://' + re.compile(r'''(www\.dreams-travel\.com.*?html?)''', re.X|re.S).search(div).group(1)
                        #行程类型
                        route.type =vacation_type  
                        if route.title.find('自由行')>-1:
                            route.type = '自由行'
                        #行程天数
                        days_temp = re.compile(r'''(\d+)(?:天|日)''', re.X|re.S).search(route.title)
                        if days_temp:
                            route.days = days_temp.group(1)    
  
                        #出发城市
                        outcityTemp = re.compile(u'''出发地：\s*([^<>\s]+)''', re.X|re.S).search(div.decode("utf-8"))
                        if outcityTemp:
                            route.outcity = outcityTemp.group(1).strip()
                        
                        print route.rout_url
                        route.rout_url = urllib.quote(route.rout_url,'utf-8')
                        route.rout_url = route.rout_url.replace('%2F','/').replace('%3A',':')       
                        innerHtml = urllib2.urlopen(route.rout_url).read()
                        innerHtml = innerHtml.decode("gb18030",'ignore').encode('utf-8')
    
                        #价格
                        priceTemp = re.compile(r'''<input\s*name="jg_cr"\s*type=hidden\s*value='?"?(\d+)''', re.X|re.S).search(innerHtml)
                        if priceTemp:
                            route.price = priceTemp.group(1)
                        else:
                            continue
                        #出发交通工具
                        route.go_t = '飞机'
                        #回程交通工具
                        route.back_t = '飞机'
                                    
                        #进一步解析 
                        do_parse(innerHtml,route)
                    except:
                        continue
                    print >>sys.stderr, 'updating:', route.supplier,'， 价格：',route.price, '， 天数：',route.days,'， 出发日期：',route.dates,'， 出发城市：',route.outcity,'， 到达城市：',route.dest,'， title：',route.title
                    
                    route.updRoute()
                    route.updRoute_Schedules()


    except:
        print traceback.format_exc(sys.exc_info())

'''
    功能：日期转化
    输入为： 2010-6-6;7-8;8-8 或者中文，类似 ‘每周四’
    8/7,8/14,8/21,1/28
    输出为：2010-08-07;2010-08-14;2010-08-21;2011-01-28
''' 
def getDates(str):

    if str.find('天天')>=0 or str.find('每')>=0 or str.find('协商')>=0:
        return '天天发团'
    elif str.find('-')<0 and (not str.isdigit()):
        return ''
    li = [];
    now = datetime.datetime.now()
    dates_p = re.compile(r''';|,|''', re.X|re.S);    
    dateList = dates_p.split(str)
    year=2010
    mon=0
    day=0
    for dateStr in dateList:
        try:
            list = dateStr.split('-')
            if(len(list)==3):
                year,mon,day = int(list[0]),int(list[1]),int(list[2])
            elif(len(list)==2):
                mon,day = int(list[0]),int(list[1])
            else:
                day = int(list[0])
            theDay = datetime.datetime(year,mon,day)
            if theDay < now:
                theDay = datetime.datetime(now.year+1,mon,day)
            ss = theDay.strftime('%Y-%m-%d')
            li.append(ss)
        except:
            print >>sys.stderr, '被忽略的日期：',dateStr
            continue
        
    return ";".join(li)

#解析每一天的行程安排----度假
def do_parse(html,route): 
    re_h=re.compile('</?\w+[^>]*>')#HTML标签
    #简介
    detail_block = re.compile(r'''(<table[^>]*?>\s*<tr>\s*<td[^>]*?><?.*?>?行程特色.*?</table>)''', re.X|re.S).search(html)
    if(detail_block):
        detail_block = detail_block.group(1)
        detail=re_h.sub('',detail_block) #去掉HTML 标签
        route.detail = detail.replace('行程特色','').replace('用心设计行程：','').replace('&nbsp;','').strip()
        
    #出发日期
    dates_temp = re.compile(u'''<input\s*name="?jg_dr"?\s*type=hidden\s*value='?"?([^"'（）]+)''', re.X|re.S).search(html.decode('utf-8'))
    if(dates_temp):
        route.dates = dates_temp.group(1)
        print route.dates 
        route.dates = route.dates.replace('年','-').replace('月','-').replace('日','').replace(' ','').replace('；',';').replace('，',';').replace('、',';')
        route.dates = getDates(route.dates)
    
    #行程
    xingcheng_block = re.compile(r'''(<table[^>]*?>\s*<tr>\s*<td[^>]*?><?[^<>]*?>?旅游行程.*?</table>)''', re.X|re.S).search(html)
    if(xingcheng_block):    
        schedule_list = re.compile(r'''<font\s*color=(?:blue|red)\s*>(.*?)(?=(?:<font\s*color=(?:blue|red)\s*>|<!--end-->|</td>))''', re.X|re.S).findall(html)
        if schedule_list:  
            route.days = len(schedule_list)
            i=0   
            for parthtml in schedule_list:
                schedule = Schedule()
                uparthtml= parthtml.strip().decode("utf-8")
        
                #第几天
                schedule.s_num = i+1
                
                #游览地点
                dest_temp = re.compile(u'''第.{1,2}天\s*([^<()（）\s:\d]+)''', re.X|re.S).search(uparthtml)
                if dest_temp:
                    dest_p = re.compile(r'''—+|－+|-+|/|\s''', re.X|re.S);    
                    schedule.s_place = dest_p.split(dest_temp.group(1).encode('utf-8').replace('&nbsp;','').replace('：','').strip())[-1]
                    if route.dest=='':
                        route.dest = schedule.s_place
                
                #详细描述
                schedule.s_brief = re_h.sub('',parthtml) #去掉HTML 标签
                schedule.s_brief = schedule.s_brief.replace('&nbsp;','').strip()
            
                #早餐： 1有、0没有 
                meals =  re.compile(r'''用餐：\s*早餐-(.*?)午餐-(.*?)晚餐-([^\s]*)''', re.X|re.S).findall(parthtml)     
                if meals:   
                    schedule.s_brf = '0' if meals[0][0].replace('&nbsp;','').strip() == '' else '1'
                    schedule.s_lunch = '0' if meals[0][1].replace('&nbsp;','').strip() == '' else '1'
                    schedule.s_dinner = '0' if meals[0][2].replace('&nbsp;','').strip() == '' else '1'
        
                hotel_temp =  re.compile(r'''酒店：\s*([^\s<>]+)''', re.X|re.S).search(parthtml)   
                #酒店描述：可选       
                if hotel_temp:
                    schedule._hotel = hotel_temp.group(1)
                route.schedules.append(schedule)
    return 

if __name__ == '__main__':
    do_crawl()
    print '---------------end---------------'
    """
    url = 'http://www.dreams-travel.com/jpxl/DR-ou-fafree08.htm'
    url = urllib.quote(url,'utf-8')
    url = url.replace('%2F','/').replace('%3A',':')       
    html = urllib2.urlopen(url).read()
    html = html.decode("GB18030",'ignore').encode('utf-8')
    print html
    """
